﻿//
// (C) Copyright 2009 Irantha Suwandarathna (iranthas@hotmail.com)
// All rights reserved.
//

/* Copyright (c) 2001-2008, The HSQL Development Group
 * All rights reserved.
 *
 * Redistribution and use _in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 *
 * Redistributions _in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer _in the documentation
 * and/or other materials provided with the distribution.
 *
 * Neither the name of the HSQL Development Group nor the names of its
 * contributors may be used to endorse or promote products derived from this
 * software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG,
 * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using EffiProz.Core.Lib;
using EffiProz.Core.DataTypes;
using EffiProz.Core.Errors;
using EffiProz.Core.Store;
using EffiProz.Core.Lib.IO;

namespace EffiProz.Core
{
    public class Scanner
    {

        /*
        <delimiter token> ::=
        <character string literal>
        | <date string>
        | <time string>
        | <timestamp string>
        | <interval string>
        | <delimited identifier>
        | <SQL special character>
        | <not equals operator>
        | <greater than or equals operator>
        | <less than or equals operator>
        | <concatenation operator>
        | <right arrow>
        | <left bracket trigraph>
        | <right bracket trigraph>
        | <double colon>
        | <double period>

        */
        //J-
        static char[] specials = new char[] {
        '"',
        '%',
        '&',
        '\'',
        '(',
        ')',
        '*',
        '+',
        ',',
        '-',
        '.',
        '/',
        '\\',
        ':',
        ';',
        '<',
        '=',
        '>',
        '?',
        '[',
        ']',
        '^',
        '_',
        '|',
        '{',
        '}'
    };

        static String[] multi = new String[] {
        "??(",
        "??)",
        "<>",
        ">=",
        "<=",
        "||",
        "->",
        "::",
        "..",
        "--",
        "/*",
        "*/",
    };

        static char[] whitespace = {
        // SQL extras
        (Char)0x9,
        (Char)0xA,
        (Char)0xB,
        (Char)0xC,
        (Char)0xD,
        (Char)0x20,
        (Char)0x85,
        // U Zs
        (Char)0x0020,
        (Char)0x00A0,
        (Char)0x1680,
        (Char)0x180E,
        (Char)0x2000,
        (Char)0x2001,
        (Char)0x2002,
        (Char)0x2003,
        (Char)0x2004,
        (Char)0x2005,
        (Char)0x2006,
        (Char)0x2007,
        (Char)0x2008,
        (Char)0x2009,
        (Char)0x200A,
        (Char)0x202F,
        (Char)0x205F,
        (Char)0x3000,
        // U Zl
        (Char)0x2028,
        // U Zp
        (Char)0x2029,
    };

        //J+
        static OrderedIntHashSet whiteSpaceSet = new OrderedIntHashSet(32);

        static Scanner()
        {
            for (int i = 0; i < whitespace.Length; i++)
            {
                whiteSpaceSet.add(whitespace[i]);
            }
        }

        // single token types
        String sqlString;
        int currentPosition;
        int tokenPosition;
        int limit;
        public Token token = new Token();

        //
        private bool hasNonSpaceSeparator;

        //
        //private static int maxPooledStringLength = 16;

        //
        char[] charBuffer = new char[256];
        CharArrayWriter charWriter;

        //
        byte[] byteBuffer = new byte[256];
        ByteArrayOutputStream byteOutputStream;


        public Scanner()
        {
            charWriter = new CharArrayWriter(charBuffer);
            byteOutputStream = new ByteArrayOutputStream(byteBuffer);
        }

        public Scanner(String sql)
        {
            charWriter = new CharArrayWriter(charBuffer);
            byteOutputStream = new ByteArrayOutputStream(byteBuffer);
            reset(sql);
        }

        public void reset(String sql)
        {

            sqlString = sql;
            currentPosition = 0;
            tokenPosition = 0;
            limit = sqlString.Length;
            hasNonSpaceSeparator = false;

            token.reset();

            token.tokenType = Tokens.X_STARTPARSE;
        }

        void resetState()
        {

            tokenPosition = currentPosition;

            token.reset();
        }

        public void scanNext()
        {

            if (currentPosition == limit)
            {
                resetState();

                token.tokenType = Tokens.X_ENDPARSE;

                return;
            }

            if (scanSeparator())
            {

                //            token.isDelimiter = true;
            }

            if (currentPosition == limit)
            {
                resetState();

                token.tokenType = Tokens.X_ENDPARSE;

                return;
            }

            bool needsDelimiter = !token.isDelimiter;

            scanToken();

            if (needsDelimiter && !token.isDelimiter)
            {

                //            token.tokenType = Token.X_UNKNOWN_TOKEN;
            }

            if (token.isMalformed)
            {
                token.fullString = getPart(tokenPosition, currentPosition);
            }
        }

        public void scanEnd()
        {

            if (currentPosition == limit)
            {
                resetState();

                token.tokenType = Tokens.X_ENDPARSE;
            }
        }

        public Token getToken()
        {
            return token;
        }

        public String getString()
        {
            return token.tokenString;
        }

        public int getTokenType()
        {
            return token.tokenType;
        }

        public Object getValue()
        {
            return token.tokenValue;
        }

        public SqlType getDataType()
        {
            return token.dataType;
        }

        public int getTokenPosition()
        {
            return tokenPosition;
        }

        public int getPosition()
        {
            return tokenPosition;
        }

        public void position(int position)
        {
            currentPosition = tokenPosition = position;
        }

        public String getPart(int start, int end)
        {
            return sqlString.Substring(start, end - start);
        }

        private int charAt(int i)
        {

            if (i >= limit)
            {
                return -1;
            }

            return sqlString[i];
        }

        void scanBinaryString()
        {

            byteOutputStream.reset(byteBuffer);

            while (true)
            {
                scanBinaryStringPart();

                if (token.isMalformed)
                {
                    return;
                }

                if (scanSeparator() && charAt(currentPosition) == '\'')
                {
                    continue;
                }

                break;
            }

            token.tokenValue = new BinaryData(byteOutputStream.toByteArray().Reverse().ToArray(),
                                              false);

            byteOutputStream.reset(byteBuffer);
        }

        /**
         * returns hex value of a hex character, or 16 if not a hex character
         */
        static int getHexValue(int c)
        {

            if (c >= '0' && c <= '9')
            {
                c -= '0';
            }
            else if (c > 'z')
            {
                c = 16;
            }
            else if (c >= 'a')
            {
                c -= ('a' - 10);
            }
            else if (c > 'Z')
            {
                c = 16;
            }
            else if (c >= 'A')
            {
                c -= ('A' - 10);
            }
            else
            {
                c = -1;
            }

            return c;
        }

        public void scanBinaryStringWithQuote()
        {

            resetState();
            scanSeparator();

            if (charAt(currentPosition) != '\'')
            {
                token.tokenType = Tokens.X_MALFORMED_BINARY_STRING;
                token.isMalformed = true;

                return;
            }

            scanBinaryString();
        }

        void scanBinaryStringPart()
        {

            bool complete = false;
            bool hi = true;
            byte b = 0;

            currentPosition++;

            for (; currentPosition < limit; currentPosition++)
            {
                int c = sqlString[currentPosition];

                if (c == ' ')
                {
                    continue;
                }

                if (c == '\'')
                {
                    complete = true;

                    currentPosition++;

                    break;
                }

                c = getHexValue(c);

                if (c == -1)
                {

                    // bad character
                    token.tokenType = Tokens.X_MALFORMED_BINARY_STRING;
                    token.isMalformed = true;

                    return;
                }

                if (hi)
                {
                    b = (byte)(c << 4);
                    hi = false;
                }
                else
                {
                    b += (byte)c;

                    byteOutputStream.writeByte(b);

                    hi = true;
                }
            }

            if (!hi)
            {

                // odd nibbles
                token.tokenType = Tokens.X_MALFORMED_BINARY_STRING;
                token.isMalformed = true;

                return;
            }

            if (!complete)
            {

                // no end quote
                token.tokenType = Tokens.X_MALFORMED_BINARY_STRING;
                token.isMalformed = true;

                return;
            }
        }

        void scanBitString()
        {

            BitMap map = new BitMap(32);

            while (true)
            {
                scanBitStringPart(map);

                if (token.isMalformed)
                {
                    return;
                }

                if (scanSeparator() && charAt(currentPosition) == '\'')
                {
                    continue;
                }

                break;
            }

            token.tokenValue = BinaryData.getBitData(map.getBytes(), map.size());
        }

        public void scanBitStringWithQuote()
        {

            resetState();
            scanSeparator();

            if (charAt(currentPosition) != '\'')
            {
                token.tokenType = Tokens.X_MALFORMED_BIT_STRING;
                token.isMalformed = true;

                return;
            }

            scanBitString();
        }

        void scanBitStringPart(BitMap map)
        {

            bool complete = false;
            int bitIndex = map.size();

            currentPosition++;
            BitMap tmpMap = new BitMap(32);

            for (; currentPosition < limit; currentPosition++)
            {
                char c = sqlString[currentPosition];

                if (c == ' ')
                {
                    continue;
                }

                if (c == '\'')
                {
                    complete = true;

                    currentPosition++;

                    break;
                }

                if (c == '0')
                {
                    bitIndex++;
                }
                else if (c == '1')
                {
                    tmpMap.set(bitIndex);

                    bitIndex++;
                }
                else
                {
                    token.tokenType = Tokens.X_MALFORMED_BIT_STRING;
                    token.isMalformed = true;
                    
                    return;
                }
            }

            if (!complete)
            {
                token.tokenType = Tokens.X_MALFORMED_BIT_STRING;
                token.isMalformed = true;

                return;
            }

          
            for (int i = 0; i < bitIndex; i++)
            {
                if (tmpMap.isSet(i))
                    map.set(bitIndex - 1 -i);
            }

            map.setSize(bitIndex);
        }

        void convertUnicodeString(int escape)
        {

            charWriter.reset(charBuffer);

            int position = 0;

            for (; ; )
            {
                int nextIndex = token.tokenString.IndexOf((char)escape, position);

                if (nextIndex < 0)
                {
                    nextIndex = token.tokenString.Length;
                }

                charWriter.write(token.tokenString, position,
                                 nextIndex - position);

                if (nextIndex == token.tokenString.Length)
                {
                    break;
                }

                nextIndex++;

                if (nextIndex == token.tokenString.Length)
                {
                    token.tokenType = Tokens.X_MALFORMED_UNICODE_STRING;
                    token.isMalformed = true;

                    return;
                }

                if (token.tokenString[nextIndex] == escape)
                {
                    charWriter.write(escape);

                    nextIndex++;

                    position = nextIndex;

                    continue;
                }

                if (nextIndex > token.tokenString.Length - 4)
                {
                    token.tokenType = Tokens.X_MALFORMED_UNICODE_STRING;
                    token.isMalformed = true;

                    return;
                }

                int hexCount = 4;
                int hexIndex = 0;
                int hexValue = 0;

                if (token.tokenString[nextIndex] == '+')
                {
                    nextIndex++;

                    if (nextIndex > token.tokenString.Length - 6)
                    {
                        token.tokenType = Tokens.X_MALFORMED_UNICODE_STRING;
                        token.isMalformed = true;

                        return;
                    }

                    hexIndex = 2;
                    hexCount = 8;
                }

                for (; hexIndex < hexCount; hexIndex++)
                {
                    int character = token.tokenString[position++];

                    character = getHexValue(character);

                    if (character == -1)
                    {
                        token.tokenType = Tokens.X_MALFORMED_UNICODE_STRING;
                        token.isMalformed = true;

                        return;
                    }

                    hexValue |= character << ((hexCount - hexIndex - 1) * 4);
                }

                if (hexCount == 8)
                {
                    charWriter.write((int)((uint)hexValue >> 16));
                }

                charWriter.write(hexValue & (hexValue & 0xffff));

                token.tokenValue = charWriter.toString();
            }
        }

        /**
         * Only for identifiers that are part of known token sequences
         */
        bool scanSpecialIdentifier(String identifier)
        {

            int length = identifier.Length;

            if (limit - currentPosition < length)
            {
                return false;
            }

            for (int i = 0; i < length; i++)
            {
                int character = identifier[i];

                if (character == sqlString[currentPosition + i])
                {
                    continue;
                }

                if (character
                        == Char.ToUpper(sqlString[currentPosition
                            + i]))
                {
                    continue;
                }

                return false;
            }

            currentPosition += length;

            return true;
        }

        private int scanEscapeDefinition()
        {

            int c = charAt(currentPosition);

            if (c == '\'')
            {
                currentPosition++;

                if (!scanWhitespace())
                {
                    c = charAt(currentPosition);

                    if (getHexValue(c) == -1)
                    {
                        if (c != '+' && c != '\'' && c != '\"')
                        {
                            int escape = c;

                            currentPosition++;

                            c = charAt(currentPosition);

                            if (c == '\'')
                            {
                                currentPosition++;

                                return escape;
                            }
                        }
                    }
                }
            }

            return -1;
        }

        private void scanUnicodeString()
        {

            int escape = '\\';

            scanCharacterString();
            scanSeparator();

            int c = charAt(currentPosition);

            if (c == 'u' || c == 'U')
            {
                if (scanSpecialIdentifier(Tokens.T_UESCAPE))
                {
                    scanSeparator();

                    escape = scanEscapeDefinition();

                    if (escape == -1)
                    {
                        token.tokenType = Tokens.X_MALFORMED_UNICODE_ESCAPE;
                        token.isMalformed = true;

                        return;
                    }
                }
            }

            convertUnicodeString(escape);
        }

        private bool scanUnicodeIdentifier()
        {

            int escape = '\\';

            scanStringPart('"');

            if (token.isMalformed)
            {
                return false;
            }

            token.tokenString = charWriter.toString();

            int c = charAt(currentPosition);

            if (c == 'u' || c == 'U')
            {
                if (scanSpecialIdentifier(Tokens.T_UESCAPE))
                {
                    scanSeparator();

                    escape = scanEscapeDefinition();

                    if (escape == -1)
                    {
                        token.tokenType = Tokens.X_MALFORMED_UNICODE_ESCAPE;
                        token.isMalformed = true;

                        return false;
                    }
                }
            }

            convertUnicodeString(escape);

            return !token.isMalformed;
        }

        bool shiftPrefixes()
        {

            if (token.namePrePrePrefix != null)
            {
                return false;
            }

            token.namePrePrePrefix = token.namePrePrefix;
            token.isDelimitedPrePrePrefix = token.isDelimitedPrePrefix;
            token.namePrePrefix = token.namePrefix;
            token.isDelimitedPrePrefix = token.isDelimitedPrefix;
            token.namePrefix = token.tokenString;
            token.isDelimitedPrefix = (token.tokenType
                                       == Tokens.X_DELIMITED_IDENTIFIER);

            return true;
        }

        private void scanIdentifierChain()
        {

            int c = charAt(currentPosition);

            switch (c)
            {

                case '"':
                    charWriter.reset(charBuffer);
                    scanStringPart('"');

                    if (token.isMalformed)
                    {
                        return;
                    }

                    token.tokenType = Tokens.X_DELIMITED_IDENTIFIER;
                    string tokenString_tmp = charWriter.toString();
                    token.tokenString = tokenString_tmp.ToUpper(System.Globalization.CultureInfo.InvariantCulture);
                    token.tokenStringOrginal = tokenString_tmp;
                    token.isDelimiter = true;
                    break;

                case 'u':
                case 'U':
                    if (charAt(currentPosition + 1) == '&')
                    {
                        if (charAt(currentPosition + 1) == '"')
                        {
                            currentPosition += 3;

                            bool result = scanUnicodeIdentifier();

                            if (!result)
                            {
                                return;
                            }

                            token.tokenType = Tokens.X_DELIMITED_IDENTIFIER;
                            token.isDelimiter = false;

                            break;
                        }
                    }
                    goto default;
                    
                // fall through
                default:
                    bool result1 = scanUndelimitedIdentifier();

                    if (!result1)
                    {
                        return;
                    }

                    token.tokenType = Tokens.X_IDENTIFIER;
                    token.isDelimiter = false;
                    break;
            }

            c = charAt(currentPosition);

            if (c == '.')
            {
                currentPosition++;

                c = charAt(currentPosition);

                if (c == '*')
                {
                    currentPosition++;

                    shiftPrefixes();

                    token.tokenString = Tokens.T_ASTERISK;
                    token.tokenType = Tokens.ASTERISK;
                }
                else
                {
                    shiftPrefixes();
                    scanIdentifierChain();
                }
            }
        }

        public bool scanUndelimitedIdentifier()
        {

            if (currentPosition == limit)
            {
                return false;
            }

            char start = sqlString[currentPosition];

            if (!Char.IsLetter(start) && !(token.isHostParameter && start == '@'))
            {
                token.tokenString = start.ToString();
                token.tokenType = Tokens.X_UNKNOWN_TOKEN;
                token.isMalformed = true;

                return false;
            }

            int i = currentPosition + 1;

            for (; i < limit; i++)
            {
                char c = sqlString[i];

                if (c == '_' || Char.IsLetterOrDigit(c))
                {
                    continue;
                }

                break;
            }

            string tmp = sqlString.Substring(currentPosition,
                    i - currentPosition);
            token.tokenString = tmp.ToUpper(System.Globalization.CultureInfo.InvariantCulture);
            token.tokenStringOrginal = tmp;
            currentPosition = i;


            int tokenLength = currentPosition - tokenPosition;

            //if (tokenLength == 4 || tokenLength == 5)
            //{
            //    switch (start)
            //    {

            //        case 'T':
            //        case 't':
            //            if (Tokens.T_TRUE.Equals(token.tokenString,StringComparison.OrdinalIgnoreCase))
            //            {
            //                token.tokenString = Tokens.T_TRUE;
            //                token.tokenType = Tokens.TRUE;
            //                token.tokenValue = true;
            //                token.dataType = SqlType.SQL_BOOLEAN;
            //                token.isDelimiter = false;
            //                return false;
            //            }
            //            break;

            //        case 'F':
            //        case 'f':
            //            if (Tokens.T_FALSE.Equals(token.tokenString, StringComparison.OrdinalIgnoreCase))
            //            {
            //                token.tokenString = Tokens.T_FALSE;
            //                token.tokenType = Tokens.FALSE;
            //                token.tokenValue = false;
            //                token.dataType = SqlType.SQL_BOOLEAN;
            //                token.isDelimiter = false;
            //                return false;
            //            }
            //            break;

            //        case 'N':
            //        case 'n':
            //            if (Tokens.T_NULL.Equals(token.tokenString, StringComparison.OrdinalIgnoreCase))
            //            {
            //                token.tokenString = Tokens.T_NULL;
            //                token.tokenType = Tokens.X_VALUE;
            //                token.tokenValue = null;

            //                return false;
            //            }
            //            break;
            //    }
            //}

            return true;
        }

        void scanNumber()
        {

            int c;
            bool hasDigit = false;
            bool hasPoint = false;
            int exponentIndex = -1;

            token.tokenType = Tokens.X_VALUE;
            token.dataType = SqlType.SQL_INTEGER;

            int tokenStart = currentPosition;

            for (; currentPosition < limit; currentPosition++)
            {
                bool end = false;

                c = charAt(currentPosition);

                switch (c)
                {

                    case '0':
                    case '1':
                    case '2':
                    case '3':
                    case '4':
                    case '5':
                    case '6':
                    case '7':
                    case '8':
                    case '9':
                        hasDigit = true;
                        break;

                    case '.':
                        token.dataType = SqlType.SQL_NUMERIC;

                        if (hasPoint || exponentIndex != -1)
                        {
                            token.tokenString = sqlString.Substring(tokenStart,
                                    currentPosition + 1 - tokenStart);                          
                            token.tokenType = Tokens.X_MALFORMED_NUMERIC;
                            token.isMalformed = true;

                            return;
                        }

                        hasPoint = true;
                        break;

                    case 'E':
                    case 'e':
                        token.dataType = SqlType.SQL_DOUBLE;

                        if (exponentIndex != -1 || !hasDigit)
                        {
                            token.tokenString = sqlString.Substring(tokenStart,
                                    currentPosition + 1 - tokenStart);                            
                            token.tokenType = Tokens.X_MALFORMED_NUMERIC;
                            token.isMalformed = true;

                            return;
                        }

                        hasPoint = true;
                        exponentIndex = currentPosition;
                        break;

                    case '-':
                    case '+':
                        if (exponentIndex != currentPosition - 1)
                        {
                            end = true;
                        }
                        break;

                    case 'K':
                    case 'k':
                    case 'M':
                    case 'm':
                    case 'G':
                    case 'g':
                    case 'T':
                    case 't':
                    case 'P':
                    case 'p':
                        if (!hasDigit || hasPoint)
                        {
                            token.tokenType = Tokens.X_MALFORMED_NUMERIC;
                            token.isMalformed = true;

                            return;
                        }

                        String s = ((char)c).ToString().ToUpper(
                            System.Globalization.CultureInfo.InvariantCulture);

                        token.lobMultiplierType = Tokens.getNonKeywordID(s,
                                Tokens.X_MALFORMED_NUMERIC);

                        if (token.lobMultiplierType
                                == Tokens.X_MALFORMED_NUMERIC)
                        {
                            token.tokenType = Tokens.X_MALFORMED_NUMERIC;
                            token.isMalformed = true;

                            return;
                        }

                        try
                        {
                            token.tokenValue =
                                int.Parse(
                                    sqlString.Substring(
                                        tokenStart, currentPosition - tokenStart));
                            token.tokenType = Tokens.X_LOB_SIZE;

                            currentPosition++;

                            token.fullString = getPart(tokenPosition,
                                                       currentPosition);
                        }
                        catch (FormatException )
                        {
                            token.tokenType = Tokens.X_MALFORMED_NUMERIC;
                            token.isMalformed = true;
                        }

                        return;

                    default:
                        end = true;
                        break;
                }

                if (end)
                {
                    break;
                }
            }

            token.tokenString = sqlString.Substring(tokenStart, currentPosition - tokenStart);

            switch (token.dataType.typeCode)
            {

                case Types.SQL_INTEGER:

                    // fredt -  -Integer.MIN_VALUE or -Long.MIN_VALUE are promoted
                    // to a wider type.
                    if (token.tokenString.Length < 11)
                    {
                        try
                        {
                            token.tokenValue =
                                int.Parse(token.tokenString);

                            return;
                        }
                        catch (Exception ) { }
                    }

                    if (this.token.tokenString.Length < 20)
                    {
                        try
                        {
                            token.dataType = SqlType.SQL_BIGINT;
                            token.tokenValue =
                                long.Parse(token.tokenString);

                            return;
                        }
                        catch (Exception ) { }
                    }

                    token.dataType = SqlType.SQL_NUMERIC;
                    goto case Types.SQL_NUMERIC;
                // fall through
                case Types.SQL_NUMERIC:
                    try
                    {
                        Decimal _decimal = Decimal.Parse(token.tokenString);

                        token.tokenValue = _decimal;
                        token.dataType = NumberType.getNumberType(Types.NUMERIC,
                               NumberType.defaultNumericPrecision, NumberType.defaultNumericScale);
                    }
                    catch (Exception )
                    {
                        token.tokenType = Tokens.X_MALFORMED_NUMERIC;
                        token.isMalformed = true;

                        return;
                    }

                    return;

                case Types.SQL_DOUBLE:
                    try
                    {
                        double d = double.Parse(token.tokenString);

                        token.tokenValue = d;
                    }
                    catch (Exception)
                    {
                        token.tokenType = Tokens.X_MALFORMED_NUMERIC;
                        token.isMalformed = true;

                        return;
                    }

                    return;
            }
        }

        bool scanSeparator()
        {

            bool result = false;

            while (true)
            {
                bool whiteSpace = scanWhitespace();

                result |= whiteSpace;

                if (scanCommentAsInlineSeparator())
                {
                    result = true;
                    hasNonSpaceSeparator = true;

                    continue;
                }

                break;
            }

            //        token.isDelimiter |= result;
            return result;
        }

        bool scanCommentAsInlineSeparator()
        {

            int character = charAt(currentPosition);

            if (character == '-' && charAt(currentPosition + 1) == '-')
            {
                int pos = sqlString.IndexOf('\r', currentPosition + 2);

                if (pos == -1)
                {
                    pos = sqlString.IndexOf('\n', currentPosition + 2);
                }
                else if (charAt(pos + 1) == '\n')
                {
                    pos++;
                }

                if (pos == -1)
                {
                    currentPosition = limit;
                }
                else
                {
                    currentPosition = pos + 1;
                }

                return true;
            }
            else if (character == '/' && charAt(currentPosition + 1) == '*')
            {
                int pos = sqlString.IndexOf("*/", currentPosition + 2);

                if (pos == -1)
                {
                    token.tokenString = sqlString.Substring(currentPosition,
                            currentPosition + 2 - currentPosition);
                    token.tokenType = Tokens.X_MALFORMED_COMMENT;
                    token.isMalformed = true;

                    return false;
                }

                currentPosition = pos + 2;

                return true;
            }

            return false;
        }

        public bool scanWhitespace()
        {

            bool result = false;

            for (; currentPosition < limit; currentPosition++)
            {
                char c = sqlString[currentPosition];

                if (c == ' ')
                {
                    result = true;

                    continue;
                }

                if (whiteSpaceSet.contains(c))
                {
                    hasNonSpaceSeparator = true;
                    result = true;

                    continue;
                }

                break;
            }

            return result;
        }

        void scanCharacterString()
        {

            charWriter.reset(charBuffer);

            while (true)
            {
                scanStringPart('\'');

                if (token.isMalformed)
                {
                    return;
                }

                if (scanSeparator() && charAt(currentPosition) == '\'')
                {
                    continue;
                }

                break;
            }

            token.tokenString = charWriter.toString();
            token.tokenValue = token.tokenString;
        }

        public void scanStringPart(char quoteChar)
        {

            currentPosition++;

            for (; ; )
            {
                int nextIndex = sqlString.IndexOf(quoteChar, currentPosition);

                if (nextIndex < 0)
                {
                    token.tokenString = sqlString.Substring(currentPosition,
                            limit - currentPosition);
                    token.tokenType = quoteChar == '\'' ? Tokens.X_MALFORMED_STRING
                                                        : Tokens
                                                        .X_MALFORMED_IDENTIFIER;
                    token.isMalformed = true;

                    return;
                }

                if (charAt(nextIndex + 1) == quoteChar)
                {
                    nextIndex += 1;

                    charWriter.write(sqlString, currentPosition,
                                     nextIndex - currentPosition);

                    currentPosition = nextIndex + 1;

                    continue;
                }
                else
                {
                    charWriter.write(sqlString, currentPosition,
                                     nextIndex - currentPosition);

                    currentPosition = nextIndex + 1;

                    break;
                }
            }
        }

        /**
         * token [separator]  , nondelimiter {delimiter | separator}
         */
        void scanToken()
        {

            int character = charAt(currentPosition);

            resetState();

            token.tokenType = Tokens.X_IDENTIFIER;

            switch (character)
            {

                /*
                            case '%' :
                            case '^' :
                            case '&' :
                            case ':' :
                            case '{' :
                            case '}' :
                                break;
                */
                case '(':
                    token.tokenString = Tokens.T_OPENBRACKET;
                    token.tokenType = Tokens.OPENBRACKET;

                    currentPosition++;

                    token.isDelimiter = true;

                    return;

                case ')':
                    token.tokenString = Tokens.T_CLOSEBRACKET;
                    token.tokenType = Tokens.CLOSEBRACKET;

                    currentPosition++;

                    token.isDelimiter = true;

                    return;

                case ',':
                    token.tokenString = Tokens.T_COMMA;
                    token.tokenType = Tokens.COMMA;

                    currentPosition++;

                    token.isDelimiter = true;

                    return;

                case '*':
                    token.tokenString = Tokens.T_ASTERISK;
                    token.tokenType = Tokens.ASTERISK;

                    currentPosition++;

                    token.isDelimiter = true;

                    return;

                case '=':
                    token.tokenString = Tokens.T_EQUALS;
                    token.tokenType = Tokens.EQUALS;

                    currentPosition++;

                    token.isDelimiter = true;

                    return;

                case ';':
                    token.tokenString = Tokens.T_SEMICOLON;
                    token.tokenType = Tokens.SEMICOLON;

                    currentPosition++;

                    token.isDelimiter = true;

                    return;

                case '+':
                    token.tokenString = Tokens.T_PLUS;
                    token.tokenType = Tokens.PLUS;

                    currentPosition++;

                    token.isDelimiter = true;

                    return;

                case ':':
                    if (charAt(currentPosition + 1) == ':')
                    {
                        currentPosition += 2;
                        token.tokenString = Tokens.T_DOUBLE_COLON;
                        token.tokenType = Tokens.COLON;
                        token.isDelimiter = true;

                        return;
                    }
                    else
                    {
                        token.tokenString = Tokens.T_COLON;
                        token.tokenType = Tokens.COLON;

                        currentPosition++;

                        token.isDelimiter = true;

                        return;
                    }
                case '@':                   
                    //currentPosition++;
                    token.isHostParameter = true;
                    break;
           
                case '?':
                    if (charAt(currentPosition + 1) == '?')
                    {
                        if (charAt(currentPosition + 2) == '(')
                        {
                            token.tokenString = Tokens.T_OPENBRACKET;
                            token.tokenType = Tokens.OPENBRACKET;
                            currentPosition += 3;
                            token.isDelimiter = true;

                            return;
                        }
                        else if (charAt(currentPosition + 2) == ')')
                        {
                            token.tokenString = Tokens.T_CLOSEBRACKET;
                            token.tokenType = Tokens.CLOSEBRACKET;
                            currentPosition += 3;
                            token.isDelimiter = true;

                            return;
                        }
                    }

                    token.tokenString = Tokens.T_QUESTION;
                    token.tokenType = Tokens.QUESTION;

                    currentPosition++;

                    token.isDelimiter = true;

                    return;

                case '!':
                    if (charAt(currentPosition + 1) == '=')
                    {
                        token.tokenString = Tokens.T_NOT_EQUALS;
                        token.tokenType = Tokens.NOT_EQUALS;
                        currentPosition += 2;
                        token.isDelimiter = true;

                        return;
                    }

                    token.tokenString = sqlString.Substring(currentPosition,
                            currentPosition + 2 - currentPosition);
                    token.tokenType = Tokens.X_UNKNOWN_TOKEN;
                    token.isDelimiter = true;

                    return;

                case '<':
                    if (charAt(currentPosition + 1) == '>')
                    {
                        token.tokenString = Tokens.T_NOT_EQUALS;
                        token.tokenType = Tokens.NOT_EQUALS;
                        currentPosition += 2;
                        token.isDelimiter = true;

                        return;
                    }

                    if (charAt(currentPosition + 1) == '=')
                    {
                        token.tokenString = Tokens.T_LESS_EQUALS;
                        token.tokenType = Tokens.LESS_EQUALS;
                        currentPosition += 2;
                        token.isDelimiter = true;

                        return;
                    }

                    token.tokenString = Tokens.T_LESS;
                    token.tokenType = Tokens.LESS;

                    currentPosition++;

                    token.isDelimiter = true;

                    return;

                case '>':
                    if (charAt(currentPosition + 1) == '=')
                    {
                        token.tokenString = Tokens.T_GREATER_EQUALS;
                        token.tokenType = Tokens.GREATER_EQUALS;
                        currentPosition += 2;
                        token.isDelimiter = true;

                        return;
                    }

                    token.tokenString = Tokens.T_GREATER;
                    token.tokenType = Tokens.GREATER;

                    currentPosition++;

                    token.isDelimiter = true;

                    return;

                case '|':
                    if (charAt(currentPosition + 1) == '|')
                    {
                        token.tokenString = Tokens.T_CONCAT;
                        token.tokenType = Tokens.CONCAT;
                        currentPosition += 2;
                        token.isDelimiter = true;

                        return;
                    }

                    token.tokenString = sqlString.Substring(currentPosition,
                            currentPosition + 2 - currentPosition);
                    token.tokenType = Tokens.X_UNKNOWN_TOKEN;
                    token.isDelimiter = true;

                    return;

                case '/':
                    if (charAt(currentPosition + 1) == '/')
                    {
                        int pos = sqlString.IndexOf('\r', currentPosition + 2);

                        if (pos == -1)
                        {
                            pos = sqlString.IndexOf('\n', currentPosition + 2);
                        }

                        if (pos == -1)
                        {
                            pos = limit;
                        }

                        token.tokenString = sqlString.Substring(currentPosition
                                + 2, pos - (currentPosition
                                + 2));
                        token.tokenType = Tokens.X_REMARK;
                        token.isDelimiter = true;

                        return;
                    }
                    else if (charAt(currentPosition + 1) == '*')
                    {
                        int pos = sqlString.IndexOf("*/", currentPosition + 2);

                        if (pos == -1)
                        {
                            token.tokenString =
                                sqlString.Substring(currentPosition,
                                                    currentPosition + 2);
                            token.tokenType = Tokens.X_UNKNOWN_TOKEN;
                            token.isDelimiter = true;

                            return;
                        }

                        token.tokenString = sqlString.Substring(currentPosition
                                + 2, pos - (currentPosition + 2));
                        token.tokenType = Tokens.X_REMARK;
                        token.isDelimiter = true;

                        return;
                    }

                    token.tokenString = Tokens.T_DIVIDE;
                    token.tokenType = Tokens.DIVIDE;

                    currentPosition++;

                    token.isDelimiter = true;

                    return;

                case '-':
                    if (charAt(currentPosition + 1) == '-')
                    {
                        int pos = sqlString.IndexOf('\r', currentPosition + 2);

                        if (pos == -1)
                        {
                            pos = sqlString.IndexOf('\n', currentPosition + 2);
                        }

                        if (pos == -1)
                        {
                            pos = limit;
                        }

                        token.tokenString = sqlString.Substring(currentPosition
                                + 2, pos - (currentPosition + 2));
                        token.tokenType = Tokens.X_REMARK;
                        token.isDelimiter = true;

                        return;
                    }

                    token.tokenString = Tokens.T_MINUS;
                    token.tokenType = Tokens.MINUS;

                    currentPosition++;

                    token.isDelimiter = true;

                    return;

                case '\"':
                    token.tokenType = Tokens.X_DELIMITED_IDENTIFIER;
                    break;

                case '\'':
                    scanCharacterString();

                    if (token.isMalformed)
                    {
                        return;
                    }

                    token.dataType = CharacterType.getCharacterType(Types.SQL_CHAR,
                            token.tokenString.Length);
                    token.tokenType = Tokens.X_VALUE;
                    token.isDelimiter = true;

                    return;

                case 'x':
                case 'X':
                    if (charAt(currentPosition + 1) == '\'')
                    {
                        currentPosition++;

                        scanBinaryString();

                        if (token.isMalformed)
                        {
                            return;
                        }

                        token.dataType = BinaryType.getBinaryType(
                            Types.SQL_VARBINARY,
                            ((BinaryData)token.tokenValue).length(null));
                        token.tokenType = Tokens.X_VALUE;

                        return;
                    }
                    break;

                case 'b':
                case 'B':
                    if (charAt(currentPosition + 1) == '\'')
                    {
                        currentPosition++;

                        scanBitString();

                        if (token.isMalformed)
                        {
                            return;
                        }

                        token.dataType = BitType.getBitType(
                            Types.SQL_BIT,
                            ((BinaryData)token.tokenValue).bitLength(null));
                        token.tokenType = Tokens.X_VALUE;

                        return;
                    }
                    break;

                case 'n':
                case 'N':
                    if (charAt(currentPosition + 1) == '\'')
                    {
                        currentPosition++;

                        scanCharacterString();

                        if (token.isMalformed)
                        {
                            return;
                        }

                        token.dataType = CharacterType.getCharacterType(
                            Types.SQL_CHAR, token.tokenString.Length);
                        token.tokenType = Tokens.X_VALUE;

                        return;
                    }
                    break;

                case 'u':
                case 'U':
                    if (charAt(currentPosition + 1) == '&')
                    {
                        if (charAt(currentPosition + 2) == '\'')
                        {
                            currentPosition += 2;
                            token.dataType = SqlType.SQL_CHAR;
                            token.tokenType = Tokens.X_VALUE;

                            scanUnicodeString();

                            if (token.isMalformed)
                            {
                                return;
                            }

                            token.dataType = CharacterType.getCharacterType(
                                Types.SQL_CHAR,
                                ((String)token.tokenValue).Length);

                            return;
                        }
                    }
                    break;

                case '_':

                    /**
                     * @todo 1.9.0 - review following
                     * identifier chain must not have catalog identifier
                     * character set specification to be included in the token.dataType
                     */
                    currentPosition++;

                    scanIdentifierChain();

                    if (token.isMalformed)
                    {
                        return;
                    }

                    if (token.tokenType != Tokens.X_IDENTIFIER
                            || token.namePrePrefix != null)
                    {

                        /** @todo 1.9.0 - review message malformed character set identifier */
                        token.tokenType = Tokens.X_MALFORMED_STRING;
                        token.isMalformed = true;

                        return;
                    }

                    token.charsetSchema = token.namePrefix;
                    token.charsetName = token.tokenString;

                    scanSeparator();

                    if (charAt(currentPosition) == '\'')
                    {
                        scanCharacterString();

                        token.tokenType = Tokens.X_VALUE;
                        token.dataType = CharacterType.getCharacterType(
                            Types.SQL_CHAR, token.tokenString.Length);
                        token.isDelimiter = true;

                        return;
                    }
                    break;

                case '0':
                case '1':
                case '2':
                case '3':
                case '4':
                case '5':
                case '6':
                case '7':
                case '8':
                case '9':
                case '.':
                    token.tokenType = Tokens.X_VALUE;

                    scanNumber();
                    return;
            }

            scanIdentifierChain();

            if (token.tokenType == Tokens.X_IDENTIFIER)
            {
                token.isUndelimitedIdentifier = true;
                token.tokenType = Tokens.getKeywordID(token.tokenString,
                                                      Tokens.X_IDENTIFIER);

                if (token.tokenType == Tokens.X_IDENTIFIER)
                {
                    token.tokenType = Tokens.getNonKeywordID(token.tokenString,
                            Tokens.X_IDENTIFIER);
                }
                else
                {
                    token.isReservedIdentifier = true;
                    token.isCoreReservedIdentifier =
                        Tokens.isCoreKeyword(token.tokenType);
                }
            }
            else if (token.tokenType == Tokens.X_DELIMITED_IDENTIFIER)
            {
                token.isDelimitedIdentifier = true;
            }
        }

        public bool scanNull()
        {

            int character = charAt(currentPosition);

            if (character == 'N' || character == 'n')
            {
                if (scanSpecialIdentifier(Tokens.T_NULL))
                {
                    return true;
                }
            }

            return false;
        }

        //
        private void scanNext(int error)
        {

            scanNext();

            if (token.isMalformed)
            {
                throw Error.error(error);
            }
        }

        /**
         * Reads the type part of the INTERVAL
         */
        IntervalType scanIntervalType()
        {

            int precision = -1;
            int scale = -1;
            int startToken;
            int endToken;
            int errorCode = ErrorCode.X_22006;

            startToken = endToken = token.tokenType;

            scanNext(errorCode);

            if (token.tokenType == Tokens.OPENBRACKET)
            {
                scanNext(errorCode);

                if (token.dataType == null
                        || token.dataType.typeCode != Types.SQL_INTEGER)
                {
                    throw Error.error(errorCode);
                }

                precision = Convert.ToInt32(this.token.tokenValue);

                scanNext(errorCode);

                if (token.tokenType == Tokens.COMMA)
                {
                    if (startToken != Tokens.SECOND)
                    {
                        throw Error.error(errorCode);
                    }

                    scanNext(errorCode);

                    if (token.dataType == null
                            || token.dataType.typeCode != Types.SQL_INTEGER)
                    {
                        throw Error.error(errorCode);
                    }

                    scale = Convert.ToInt32(token.tokenValue);

                    scanNext(errorCode);
                }

                if (token.tokenType != Tokens.CLOSEBRACKET)
                {
                    throw Error.error(errorCode);
                }

                scanNext(errorCode);
            }

            if (token.tokenType == Tokens.TO)
            {
                scanNext(errorCode);

                endToken = token.tokenType;

                scanNext(errorCode);
            }

            if (token.tokenType == Tokens.OPENBRACKET)
            {
                if (endToken != Tokens.SECOND || endToken == startToken)
                {
                    throw Error.error(errorCode);
                }

                scanNext(errorCode);

                if (token.dataType == null
                        || token.dataType.typeCode != Types.SQL_INTEGER)
                {
                    throw Error.error(errorCode);
                }

                scale = Convert.ToInt32(token.tokenValue);

                scanNext(errorCode);

                if (token.tokenType != Tokens.CLOSEBRACKET)
                {
                    throw Error.error(errorCode);
                }

                scanNext(errorCode);
            }

            int startIndex = ArrayUtil.find(Tokens.SQL_INTERVAL_FIELD_CODES,
                                            startToken);
            int endIndex = ArrayUtil.find(Tokens.SQL_INTERVAL_FIELD_CODES,
                                          endToken);

            return IntervalType.getIntervalType(startIndex, endIndex, precision,
                                                scale);
        }

        private String intervalString;
        private int intervalPosition;
        private int intervalPrecision;
        private int fractionPrecision;
        public SqlType dateTimeType;

        public TimestampData newDate(String s)
        {

            intervalPosition = 0;
            fractionPrecision = 0;
            dateTimeType = null;
            intervalString = s;

            scanDateParts(2);

            if (intervalPosition != s.Length)
            {
                throw Error.error(ErrorCode.X_22007);
            }

            long seconds = EfzDateTime.getDateSeconds(s);

            return new TimestampData(seconds);
        }

        /**
         * @todo 1.9.0 - review the following
         *      - misses nano fractions
         *      - misses displacement
         *      - doesn't allow single digit components
         */
        public TimestampData newTimestamp(String s)
        {

            long zoneSeconds = 0;
            long seconds;
            int fraction = 0;
            int endIndex = s.Length;
            bool negate;
            bool hasZone = false;

            intervalPosition = 0;
            fractionPrecision = 0;
            dateTimeType = null;
            intervalString = s;

            scanDateParts(5);

            try
            {
                seconds = EfzDateTime.getTimestampSeconds(s.Substring(0,
                        intervalPosition));
            }
            catch (Exception)
            {
                throw Error.error(ErrorCode.X_22007);
            }

            int position;

            fraction = scanIntervalFraction(DTIType.maxFractionPrecision);
            position = intervalPosition;
            negate = scanIntervalSign();

            if (negate || position != intervalPosition)
            {
                zoneSeconds = scanIntervalValue(SqlType.SQL_INTERVAL_HOUR_TO_MINUTE);
                hasZone = true;

                if (negate)
                {
                    zoneSeconds = -zoneSeconds;
                }
            }

            if (zoneSeconds >= DTIType.yearToSecondFactors[2]
                    || zoneSeconds > DTIType.timezoneSecondsLimit
                    || -zoneSeconds > DTIType.timezoneSecondsLimit)
            {
                throw Error.error(ErrorCode.X_22009);
            }

            if (intervalPosition != endIndex)
            {
                throw Error.error(ErrorCode.X_22007);
            }

            int type = hasZone ? Types.SQL_TIMESTAMP_WITH_TIME_ZONE
                               : Types.SQL_TIMESTAMP;

            dateTimeType = DateTimeType.getDateTimeType(type, fractionPrecision);

            if (hasZone)
            {
                seconds -= zoneSeconds;
            }

            return new TimestampData(seconds, fraction, (int)zoneSeconds);
        }

        void scanDateParts(int lastPart) {

            byte[]    separators    = DTIType.yearToSecondSeparators;
            int       i             = intervalPosition;
            int firstPart     = 0;
            int       currentPart   = firstPart;
            int       currentDigits = 0;

            for (; currentPart <= lastPart; ) {
                bool endOfPart = false;

                if (i == intervalString.Length) {
                    if (currentPart == lastPart) {
                        endOfPart = true;
                    } else {

                        // parts missing
                        throw Error.error(ErrorCode.X_22007);
                    }
                } else {
                    int character = intervalString[i];

                    if (character >= '0' && character <= '9') {
                        currentDigits++;
                        i++;
                    } else if (character == separators[currentPart]) {
                        endOfPart = true;

                        if (currentPart != lastPart) {
                            i++;
                        }
                    } else if (currentPart == lastPart) {
                        endOfPart = true;
                    } else {
                        throw Error.error(ErrorCode.X_22007);
                    }
                }

                if (endOfPart) {
                    if (currentPart == firstPart) {
                        if (currentDigits != 4) {
                            throw Error.error(ErrorCode.X_22007);
                        }
                    } else {
                        if (currentDigits != 2) {
                            throw Error.error(ErrorCode.X_22007);
                        }
                    }

                    currentPart++;

                    currentDigits = 0;

                    if (i == intervalString.Length) {
                        break;
                    }
                }
            }

            intervalPosition = i;
        }

        public TimeData newTime(String s)
        {

            intervalPosition = 0;
            fractionPrecision = 0;
            dateTimeType = null;
            intervalString = s;

            long seconds = scanIntervalValue(SqlType.SQL_INTERVAL_HOUR_TO_SECOND);
            int fraction = scanIntervalFraction(DTIType.maxFractionPrecision);
            long zoneSeconds = 0;
            int position = intervalPosition;
            bool hasZone = false;
            bool negate = scanIntervalSign();

            if (position != intervalPosition)
            {
                zoneSeconds = scanIntervalValue(SqlType.SQL_INTERVAL_HOUR_TO_MINUTE);
                hasZone = true;
            }

            if (intervalPosition != s.Length)
            {
                throw Error.error(ErrorCode.X_22009);
            }

            if (seconds >= DTIType.yearToSecondFactors[2])
            {
                throw Error.error(ErrorCode.X_22008);
            }

            if (zoneSeconds > DTIType.timezoneSecondsLimit)
            {
                throw Error.error(ErrorCode.X_22009);
            }

            if (negate)
            {
                zoneSeconds = -zoneSeconds;
            }

            int type = hasZone ? Types.SQL_TIME_WITH_TIME_ZONE
                               : Types.SQL_TIME;

            dateTimeType = DateTimeType.getDateTimeType(type, fractionPrecision);

            if (hasZone)
            {
                seconds -= zoneSeconds;
            }

            return new TimeData((int)seconds, fraction, (int)zoneSeconds);
        }

        public Object newInterval(String s, IntervalType type)
        {

            intervalPosition = 0;
            intervalString = s;

            bool negate = scanIntervalSign();
            long units = scanIntervalValue(type);
            int fraction = 0;

            if (type.endIntervalType == Types.SQL_INTERVAL_SECOND)
            {
                fraction = scanIntervalFraction(type.scale);
            }

            if (intervalPosition != s.Length)
            {
                throw Error.error(ErrorCode.X_22006);
            }

            if (negate)
            {
                units = -units;
                fraction = -fraction;
            }

            dateTimeType = type;

            if (type.defaultPrecision)
            {
                dateTimeType = IntervalType.getIntervalType(type.typeCode,
                        type.startIntervalType, type.endIntervalType,
                        intervalPrecision, fractionPrecision, false);
            }

            if (type.endPartIndex <= DTIType.INTERVAL_MONTH_INDEX)
            {
                return new IntervalMonthData(units);
            }
            else
            {
                return new IntervalSecondData(units, fraction);
            }
        }

        public long scanIntervalValue(IntervalType type)
        {

            byte[] separators = DTIType.yearToSecondSeparators;
            int[] factors = DTIType.yearToSecondFactors;
            int[] limits = DTIType.yearToSecondLimits;
            int firstPart = type.startPartIndex;
            int lastPart = type.endPartIndex;
            long totalValue = 0;
            int currentValue = 0;
            int i = intervalPosition;
            int currentPart = firstPart;
            int currentDigits = 0;

            for (; currentPart <= lastPart; )
            {
                bool endOfPart = false;

                if (i == intervalString.Length)
                {
                    if (currentPart == lastPart)
                    {
                        endOfPart = true;
                    }
                    else
                    {
                        throw Error.error(ErrorCode.X_22006);
                    }
                }
                else
                {
                    int character = intervalString[i];

                    if (character >= '0' && character <= '9')
                    {
                        int digit = character - '0';

                        currentValue *= 10;
                        currentValue += digit;

                        currentDigits++;
                        i++;
                    }
                    else if (character == separators[currentPart])
                    {
                        endOfPart = true;

                        if (currentPart != lastPart)
                        {
                            i++;
                        }
                    }
                    else if (currentPart == lastPart)
                    {
                        endOfPart = true;
                    }
                    else
                    {
                        throw Error.error(ErrorCode.X_22006);
                    }
                }

                if (endOfPart)
                {
                    if (currentPart == firstPart)
                    {
                        if (!type.defaultPrecision
                                && currentDigits > type.precision)
                        {
                            throw Error.error(ErrorCode.X_22015);
                        }

                        if (currentDigits == 0)
                        {
                            throw Error.error(ErrorCode.X_22006);
                        }

                        int factor = factors[currentPart];

                        totalValue += (long)currentValue * factor;
                        intervalPrecision = currentDigits;
                    }
                    else
                    {
                        if (currentValue >= limits[currentPart])
                        {
                            throw Error.error(ErrorCode.X_22015);
                        }

                        if (currentDigits != 2)
                        {
                            throw Error.error(ErrorCode.X_22006);
                        }

                        totalValue += currentValue * factors[currentPart];
                    }

                    currentPart++;

                    currentValue = 0;
                    currentDigits = 0;

                    if (i == intervalString.Length)
                    {
                        break;
                    }
                }
            }

            intervalPosition = i;

            return totalValue;
        }

        bool scanIntervalSign()
        {

            bool negate = false;

            if (intervalPosition == intervalString.Length)
            {
                return false;
            }

            if (intervalString[intervalPosition] == '-')
            {
                negate = true;

                intervalPosition++;
            }
            else if (intervalString[intervalPosition] == '+')
            {
                intervalPosition++;
            }

            return negate;
        }

        int scanIntervalFraction(int decimalPrecision)
        {

            if (intervalPosition == intervalString.Length)
            {
                return 0;
            }

            if (intervalString[intervalPosition] != '.')
            {
                return 0;
            }

            intervalPosition++;

            int currentValue = 0;
            int currentDigits = 0;

            for (; intervalPosition < intervalString.Length; )
            {
                int character = intervalString[intervalPosition];

                if (character >= '0' && character <= '9')
                {
                    int digit = character - '0';

                    currentValue *= 10;
                    currentValue += digit;

                    intervalPosition++;
                    currentDigits++;

                    if (currentDigits == DTIType.maxFractionPrecision)
                    {
                        break;
                    }
                }
                else
                {
                    break;
                }
            }

            fractionPrecision = currentDigits;
            currentValue *= DTIType.nanoScaleFactors[currentDigits];
            currentValue = DTIType.normaliseFraction(currentValue,
                    decimalPrecision);

            return currentValue;
        }

        void scanIntervalSpaces()
        {

            for (; intervalPosition < intervalString.Length;
                    intervalPosition++)
            {
                if (intervalString[intervalPosition] != ' ')
                {
                    break;
                }
            }
        }

        /*
         * synchronized methods for use with shared Scanner objects used for type
         *  conversion
         */
        public ValueType convertToNumber(String s,
                NumberType numberType)
        {
            lock (this)
            {
                ValueType number;
                bool minus = false;
                SqlType type;

                reset(s);
                resetState();
                scanWhitespace();
                scanToken();
                scanWhitespace();

                if (token.tokenType == Tokens.PLUS)
                {
                    scanToken();
                    scanWhitespace();
                }
                else if (token.tokenType == Tokens.MINUS)
                {
                    minus = true;

                    scanToken();
                    scanWhitespace();
                }

                if (!hasNonSpaceSeparator && token.tokenType == Tokens.X_VALUE
                        && token.tokenValue is ValueType)
                {
                    number = (ValueType)token.tokenValue;
                    type = token.dataType;

                    if (minus)
                    {
                        number = (ValueType)token.dataType.negate(number);
                    }

                    scanEnd();

                    if (token.tokenType == Tokens.X_ENDPARSE)
                    {
                        number = (ValueType)numberType.convertToType(null, number, type);

                        return number;
                    }
                }

                throw Error.error(ErrorCode.X_22018);
            }
        }

        public BinaryData convertToBinary(String s)
        {
            lock (this)
            {
                bool hi = true;
                byte b = 0;

                reset(s);
                resetState();
                byteOutputStream.reset(byteBuffer);

                for (; currentPosition < limit; currentPosition++, hi = !hi)
                {
                    int c = sqlString[currentPosition];

                    c = getHexValue(c);

                    if (c == -1)
                    {

                        // bad character
                        token.tokenType = Tokens.X_MALFORMED_BINARY_STRING;
                        token.isMalformed = true;

                        break;
                    }

                    if (hi)
                    {
                        b = (byte)(c << 4);
                    }
                    else
                    {
                        b += (byte)c;

                        byteOutputStream.writeByte(b);
                    }
                }

                if (!hi)
                {

                    // odd nibbles
                    token.tokenType = Tokens.X_MALFORMED_BINARY_STRING;
                    token.isMalformed = true;
                }

                if (token.isMalformed)
                {
                    throw Error.error(ErrorCode.X_22018);
                }

                BinaryData data = new BinaryData(byteOutputStream.toByteArray(),
                                                 false);

                byteOutputStream.reset(byteBuffer);

                return data;
            }
        }

        public BinaryData convertToBit(String s)
        {
            lock (this)
            {
                BitMap map = new BitMap(32);
                int bitIndex = map.size();

                reset(s);
                resetState();
                byteOutputStream.reset(byteBuffer);

                for (; currentPosition < limit; currentPosition++)
                {
                    int c = sqlString[currentPosition];

                    if (c == '0')
                    {
                        bitIndex++;
                    }
                    else if (c == '1')
                    {
                        map.set(bitIndex);

                        bitIndex++;
                    }
                    else
                    {
                        token.tokenType = Tokens.X_MALFORMED_BIT_STRING;
                        token.isMalformed = true;

                        throw Error.error(ErrorCode.X_22018);
                    }
                }

                map.setSize(bitIndex);

                return BinaryData.getBitData(map.getBytes(), map.size());
            }
        }

        //// should perform range checks etc.
        public Object convertToDatetimeInterval(
                SessionInterface session, String s, DTIType type)
        {
            lock (this)
            {

                Object value;
                IntervalType intervalType = null;
                int dateTimeToken = -1;
                int errorCode = type.isDateTimeType() ? ErrorCode.X_22007
                                                                   : ErrorCode.X_22006;

                reset(s);
                resetState();
                scanToken();
                scanWhitespace();

                switch (token.tokenType)
                {

                    case Tokens.INTERVAL:
                    case Tokens.DATE:
                    case Tokens.TIME:
                    case Tokens.TIMESTAMP:
                        dateTimeToken = token.tokenType;

                        scanToken();

                        if (token.tokenType != Tokens.X_VALUE
                                || token.dataType.typeCode != Types.SQL_CHAR)
                        {

                            // error datetime bad literal
                            throw Error.error(errorCode);
                        }

                        s = token.tokenString;

                        scanNext(ErrorCode.X_22007);

                        if (type.isIntervalType())
                        {
                            intervalType = scanIntervalType();
                        }

                        if (token.tokenType != Tokens.X_ENDPARSE)
                        {
                            throw Error.error(errorCode);
                        }
                        goto default;
                    // fall through
                    default:
                        break;
                }

                switch (type.typeCode)
                {

                    case Types.SQL_DATE:
                        {
                            if (dateTimeToken != -1 && dateTimeToken != Tokens.DATE)
                            {
                                throw Error.error(errorCode);
                            }

                            value = newDate(s);

                            return type.convertToType(session, value, SqlType.SQL_DATE);
                        }
                    case Types.SQL_TIME:
                    case Types.SQL_TIME_WITH_TIME_ZONE:
                        {
                            if (dateTimeToken != -1 && dateTimeToken != Tokens.TIME)
                            {
                                throw Error.error(errorCode);
                            }

                            Object o = newTime(s);

                            return type.convertToType(session, o, dateTimeType);
                        }
                    case Types.SQL_TIMESTAMP:
                    case Types.SQL_TIMESTAMP_WITH_TIME_ZONE:
                        {
                            if (dateTimeToken != -1 && dateTimeToken != Tokens.TIMESTAMP)
                            {
                                throw Error.error(errorCode);
                            }

                            value = newTimestamp(s);

                            return type.convertToType(session, value, dateTimeType);
                        }
                    default:
                        if (dateTimeToken != -1 && dateTimeToken != Tokens.INTERVAL)
                        {
                            throw Error.error(errorCode);
                        }

                        if (type.isIntervalType())
                        {
                            value = newInterval(s, (IntervalType)type);

                            if (intervalType != null)
                            {
                                if (intervalType.startIntervalType != type
                                        .startIntervalType || intervalType
                                        .endIntervalType != type.endIntervalType)
                                {
                                    throw Error.error(errorCode);
                                }
                            }

                            return type.convertToType(session, value, dateTimeType);
                        }

                        throw Error.runtimeError(ErrorCode.U_S0500, "Scanner");
                }
            }
        }
    }
}


